import datetime
import os
import Core.Gadget as Gadget
import Core.Config as Config
import Core.IO as IO
from DataProcess.Download_MutualFund_List import Download_MutualFund_List
from DataProcess.Download_MutualFund_Report import Automatic_Download_MutualFund_Reports
from DataProcess.General import Download_DailyBars
from DataProcess.Download_DailyQuote import Downloag_MutualFund_DailyQuote
import Core.WindFunctions as Wind


# 因子
# 基金公司排名 # 自行计算，可能要去除货币基金， prt_fundcototalnetassetsranking
# 基金规模 fund_fundscale -> fund_scale (report表 net asset)
# 基金可追溯历史业绩年数 track_record_year # 自行计算 即存续年数
# 基金最近连续盈利年数 profit_year # 自行计算
# 机构持有份额占比 holder_institution_holdingpct -> fund_institute_hold_proportion (report表)
# 基金经理从业时间  wind数据无日期戳，只有最新数据，fund_manager_managerworkingyears -> manager_working_years -> manager_work_year
# 基金经理从业期限内平均回报 fund_manager_geometricannualizedyield -> manager_geometric_annualized_yield -> manager_average_return
# 基金经理更换频率 自行计算，请搜索算法  fund_predfundmanager -> 算法 ->  manager_replace_freq -> manager_replace_frequency
def Automatic_Download_MutualFund(database, datetime2, startIndex=0):
    datetime1 = datetime.datetime(2015, 1, 1)

    # ---更新共募集金列表，（新基金上市，旧下市信息）---
    Download_MutualFund_List(database, forceToUpdate=True)

    # ---更新基金公司信息---
    Download_MutualFund_Company_Info(database)

    # ---更新公募基金经理信息--- 未完成
    # Download_MutualFund_Manager_Info()

    # ---必要时一起更新季度报告---
    Automatic_Download_MutualFund_Reports(database, datetime2, startIndex=0)


# 根据起始时间查找缺失Bar
def Check_MutualFund_Missing_Bar(database):

    now_datetime = datetime.datetime(2021, 4, 1)

    # Gadget.GetTradingDays_DataFrame(database, )
    # filter = [("invest_type2", "偏股混合型基金")]
    filter = [("invest_type1", "股票型基金")]

    instruments = database.Find("financial_data", "Instrument_MutualFund", filter=filter)
    instrument_count = len(instruments)
    missing_count = 0
    missing_days = 0
    i = 0
    start = False
    for instrument in instruments:
        i += 1
        if i % 100 == 0:
            print(i)

        symbol = instrument["symbol"]
        # if symbol == "020018!3.OF":
        #     start = True
        # if start == False:
        #     continue
        datetime1 = instrument["datetime1"]
        datetime2 = instrument["datetime2"]
        if datetime2 > now_datetime:
            datetime2 = now_datetime

        #
        if datetime1 < datetime.datetime(1900, 1, 1):
            continue
        if datetime1 > now_datetime:
            continue
        #
        df_bar = database.GetDataFrame("financial_data", "mutualfund_dailybar", filter=[("symbol", symbol)], sort=[("date",1)])
        if df_bar.empty:
            print(symbol + " NoData")
            if "!" in symbol:
                continue
            missing_count += 1
            num_days = (datetime2 - datetime1).days
            missing_days += num_days
            Download_DailyBars(database, symbol, instrument_type="MutualFund", begin_datetime=instrument["datetime1"], end_datetime=datetime2, logger=None, save=True)
            continue
        #
        data_datetime1 = df_bar.iloc[0]["date"]
        data_datetime2 = df_bar.iloc[-1]["date"]
        #
        gap1 = (data_datetime1 - datetime1).days
        gap2 = (datetime2 - data_datetime2).days
        # print(symbol, gap1, gap2)
        if gap1 > 10 or gap2 > 10:
            print(symbol, i, "missing data", gap1, gap2)
            missing_count += 1
            # if gap1 > 10:
            #     missing_days += gap1
            # if gap2 > 10:
            #     missing_days += gap2
        a = 0
    #
    missing_days = missing_days / 365 * 245
    print("total count", instrument_count, "missing series", missing_count, "# missing days", missing_days)


def DeleteCollectionsByList(database):
    db = database.client["MutualFund"]
    header, data = IO.ReadExcelFile("D:/Data/MutualFundLog/MissingQuote.xlsx","Exception")
    count = 0
    for d in data:
        symbol = d[header["Symbol"]]
        name = symbol + "_Time_86400_Bar"
        coll = db[name]
        count += 1
        print("Deleting Collection: " + name)
        coll.drop()
        kkwood = 0

    print("Total# " + str(count))
    kkwood = 1


def ReloadQuote(database):
    #
    header, data = IO.ReadExcelFile("D:/Data/MutualFundLog/MissingQuote.xlsx","NoData")
    instruments = []
    for d in data:
        symbol = d[header["Symbol"]]
        instruments.append(symbol)

    Wind.Batch_Request_Write_Daily_Quote(database, endDateTime, instrumentType="MutualFund",outsideInstruments=instruments )


def Download_MutualFund_Company_Info(database):
    pass
    # 基金管理人 fund_mgrcomp
    # 基金管理人简称 fund_corp_fundmanagementcompany
    # 基金数量 fund_corp_fundno
    # 团队稳定性 fund_corp_teamstability
    # 基金经理数量 fund_corp_fundmanagersno
    # 基金净资产 prt_fundcototalnetassets
    # 基金净资产排名 prt_fundcototalnetassetsranking

    fields = []
    fields.append("fund_predfundmanager")  # 历任基金经理
    fields.append("fund_corp_fundmanagementcompany")  # 基金公司简称
    pass


# 转移表格数据
def Transit_MutualFund_DailyBar(database):
    #
    from_tablename = "mutualfund_dailybar_wind"
    to_tablename = "mutualfund_dailybar"
    #
    symbols = database.ExecuteSQL("financial_data", "SELECT distinct(symbol) FROM  financial_data." + from_tablename)
    i = 0
    for symbol in symbols:
        symbol = symbol[0]
        i += 1
        documents = database.Find("financial_data", from_tablename, filter=[("symbol", symbol)])
        print(i, symbol, len(documents), datetime.datetime.now())
        #
        for document in documents:
            del document["datetime"]
            document["key2"] = document["symbol"] + "_" + Gadget.ToDateString(document["date"])

        #
        database.Insert_Many("financial_data", to_tablename, documents)
        a = 0
    a = 0


# 把20201225（28） 到 20210416 的THS数据，Merge到Wind数据后 (约76w数据)
# 操作的日期是20210603左右
# 后续有足够的数据量时候，记得删除这部分数据 where create_time = 20210603
#
# 168104
# 200001
# 10364 200002.OF duplicate
# 10365 200003.OF duplicate
# 10366 200006.OF duplicate
# 10367 200007.OF duplicate
# 10368 200008.OF duplicate
# 10369 200009.OF duplicate
# 10370 200010.OF duplicate
# 10371 200011.OF duplicate
# 10372 200012.OF duplicate
# 10373 200013.OF duplicate
# 10374 200015.OF duplicate
# 10375 200016.OF duplicate
def Merge_THS_MutualFund_DailyBar(database):
    #
    start_index = 10365
    from_tablename = "mutualfund_dailybar_2"
    to_tablename = "mutualfund_dailybar"
    #
    symbols = database.ExecuteSQL("financial_data", "SELECT distinct(symbol) FROM  financial_data." + from_tablename + " where date > 20201225")
    i = 0
    for symbol in symbols:
        symbol = symbol[0]
        i += 1
        if i < start_index:
            continue

        # documents = database.Find("financial_data", to_tablename,
        #                           filter=[("symbol", symbol), ("date", ">", "20201225")])
        # if len(documents):
        #     print(i, symbol, "duplicate")
        # continue

        documents = database.Find("financial_data", from_tablename, filter=[("symbol", symbol), ("date", ">", "20201225")])
        print(i, symbol, len(documents), datetime.datetime.now())
        #
        for document in documents:
            # del document["datetime"]
            del document["created_time"]
            document["key2"] = document["symbol"] + "_" + Gadget.ToDateString(document["date"])
        #
        database.Insert_Many("financial_data", to_tablename, documents)
        a = 0
    a = 0


def Fix_Download_Some_Bars(database):
    list = ["168104.OF"]
    list = []
    list.append("200001.OF")
    list.append("200002.OF")
    list.append("200003.OF")
    list.append("200006.OF")
    list.append("200007.OF")
    list.append("200008.OF")
    list.append("200009.OF")
    list.append("200010.OF")
    list.append("200011.OF")
    list.append("200012.OF")
    list.append("200013.OF")
    list.append("200015.OF")
    list.append("200016.OF")

    instrument_type = "MutualFund"
    end_datetime = datetime.datetime(2020, 12, 25)
    end_datetime = datetime.datetime(2016, 6, 1)

    downloader = Downloag_MutualFund_DailyQuote(database, Wind.w)

    for symbol in list:
        tmp_datetime = database.Find("financial_data", "instrument_mutualfund", filter=[("symbol", symbol)])
        tmp_datetime = tmp_datetime[0]
        begin_datetime = tmp_datetime["datetime1"]

        downloader.download_data(symbol, begin_datetime, end_datetime, save=True)
        Download_DailyBars(database, symbol, instrument_type, begin_datetime, end_datetime, logger=None, save=True)


if __name__ == '__main__':

    # ---Connecting Database---
    path_filename = os.getcwd() + "\..\Config\config_local.json"
    database = Config.create_database(database_type="MySQL", config_file=path_filename, config_field="MySQL")

    # Transit_MutualFund_DailyBar(database)
    # Merge_THS_MutualFund_DailyBar(database)

    Wind.w.start()

    # Check_MutualFund_Missing_Bar(database)
    # Fix_Download_Some_Bars(database)